<!DOCTYPE html><html lang="zh-CN"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1"><meta name="format-detection" content="telephone=no"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black"><link rel="icon" href="/images/icons/favicon-16x16.png?v=2.8.0" type="image/png" sizes="16x16"><link rel="icon" href="/images/icons/favicon-32x32.png?v=2.8.0" type="image/png" sizes="32x32"><meta name="description" content="读代码学习TensorRT         代码仓库: https:&#x2F;&#x2F;github.com&#x2F;wang-xinyu&#x2F;tensorrtx&#x2F;tree&#x2F;yolov5-v4.0&#x2F;yolov5s TensorRT 7.0 documentation: https:&#x2F;&#x2F;docs.nvidia.com&#x2F;deeplearning&#x2F;tensorrt&#x2F;archives&#x2F;tensorrt-700">
<meta property="og:type" content="article">
<meta property="og:title" content="读代码学习TensorRT">
<meta property="og:url" content="http://hipposox.github.io/2023/01/12/TensorRT/index.html">
<meta property="og:site_name" content="Hexo">
<meta property="og:description" content="读代码学习TensorRT         代码仓库: https:&#x2F;&#x2F;github.com&#x2F;wang-xinyu&#x2F;tensorrtx&#x2F;tree&#x2F;yolov5-v4.0&#x2F;yolov5s TensorRT 7.0 documentation: https:&#x2F;&#x2F;docs.nvidia.com&#x2F;deeplearning&#x2F;tensorrt&#x2F;archives&#x2F;tensorrt-700">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://www.helloimg.com/images/2023/01/14/oGD6Ac.png">
<meta property="article:published_time" content="2023-01-12T12:33:53.000Z">
<meta property="article:modified_time" content="2023-01-14T08:26:01.503Z">
<meta property="article:author" content="HippoSoX">
<meta property="article:tag" content="TensorRT">
<meta property="article:tag" content="神经网络">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://www.helloimg.com/images/2023/01/14/oGD6Ac.png"><title>读代码学习TensorRT | Hexo</title><link ref="canonical" href="http://hipposox.github.io/2023/01/12/TensorRT/"><link rel="dns-prefetch" href="https://cdn.jsdelivr.net"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.12.1/css/all.min.css" type="text/css"><link rel="stylesheet" href="/css/index.css?v=2.8.0"><link rel="stylesheet" href="css/custom.css"><script>var Stun = window.Stun || {};
var CONFIG = {
  root: '/',
  algolia: undefined,
  assistSearch: undefined,
  fontIcon: {"prompt":{"success":"fas fa-check-circle","info":"fas fa-arrow-circle-right","warning":"fas fa-exclamation-circle","error":"fas fa-times-circle"},"copyBtn":"fas fa-copy"},
  sidebar: {"offsetTop":"20px","tocMaxDepth":6},
  header: {"enable":true,"showOnPost":true,"scrollDownIcon":false},
  postWidget: {"endText":true},
  nightMode: {"enable":true},
  back2top: {"enable":true},
  codeblock: {"style":"default","highlight":"dark","wordWrap":false},
  reward: false,
  fancybox: false,
  zoomImage: {"gapAside":"20px"},
  galleryWaterfall: undefined,
  lazyload: true,
  pjax: undefined,
  externalLink: {"icon":{"enable":true,"name":"fas fa-external-link-alt"}},
  shortcuts: undefined,
  prompt: {"copyButton":"复制","copySuccess":"复制成功","copyError":"复制失败"},
  sourcePath: {"js":"js","css":"css","images":"images"},
};

window.CONFIG = CONFIG;</script><meta name="generator" content="Hexo 5.4.2"></head><body><div class="container" id="container"><header class="header" id="header"><div class="header-inner"><nav class="header-nav header-nav--fixed"><div class="header-nav-inner"><div class="header-nav-menubtn"><i class="fas fa-bars"></i></div><div class="header-nav-menu"><div class="header-nav-menu-item"><a class="header-nav-menu-item__link" href="/"><span class="header-nav-menu-item__icon"><i class="fas fa-home"></i></span><span class="header-nav-menu-item__text">首页</span></a></div><div class="header-nav-menu-item"><a class="header-nav-menu-item__link" href="/archives/"><span class="header-nav-menu-item__icon"><i class="fas fa-folder-open"></i></span><span class="header-nav-menu-item__text">归档</span></a></div><div class="header-nav-menu-item"><a class="header-nav-menu-item__link" href="/categories/"><span class="header-nav-menu-item__icon"><i class="fas fa-layer-group"></i></span><span class="header-nav-menu-item__text">分类</span></a></div><div class="header-nav-menu-item"><a class="header-nav-menu-item__link" href="/tags/"><span class="header-nav-menu-item__icon"><i class="fas fa-tags"></i></span><span class="header-nav-menu-item__text">标签</span></a></div></div><div class="header-nav-mode"><div class="mode"><div class="mode-track"><span class="mode-track-moon"></span><span class="mode-track-sun"></span></div><div class="mode-thumb"></div></div></div></div></nav><div class="header-banner"><div class="header-banner-info"><div class="header-banner-info__title">erocool</div><div class="header-banner-info__subtitle">You know what</div></div></div></div></header><main class="main" id="main"><div class="main-inner"><div class="content-wrap" id="content-wrap"><div class="content" id="content"><!-- Just used to judge whether it is an article page--><div id="is-post"></div><div class="post"><header class="post-header"><h1 class="post-title">读代码学习TensorRT</h1><div class="post-meta"><span class="post-meta-item post-meta-item--createtime"><span class="post-meta-item__icon"><i class="far fa-calendar-plus"></i></span><span class="post-meta-item__info">发表于</span><span class="post-meta-item__value">2023-01-12</span></span><span class="post-meta-item post-meta-item--updatetime"><span class="post-meta-item__icon"><i class="far fa-calendar-check"></i></span><span class="post-meta-item__info">更新于</span><span class="post-meta-item__value">2023-01-14</span></span></div></header><div class="post-body">
        <h1 id="读代码学习tensorrt"   >
          <a href="#读代码学习tensorrt" class="heading-link"><i class="fas fa-link"></i></a><a class="markdownIt-Anchor" href="#读代码学习tensorrt"></a> 读代码学习TensorRT</h1>
      
<blockquote>
<p>代码仓库: <span class="exturl"><a class="exturl__link"   target="_blank" rel="noopener" href="https://github.com/wang-xinyu/tensorrtx/tree/yolov5-v4.0/yolov5s" >https://github.com/wang-xinyu/tensorrtx/tree/yolov5-v4.0/yolov5s</a><span class="exturl__icon"><i class="fas fa-external-link-alt"></i></span></span></p>
<p>TensorRT 7.0 documentation: <span class="exturl"><a class="exturl__link"   target="_blank" rel="noopener" href="https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-700/tensorrt-api/c_api/namespacemembers_func.html" >https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-700/tensorrt-api/c_api/namespacemembers_func.html</a><span class="exturl__icon"><i class="fas fa-external-link-alt"></i></span></span></p>
<p>Cuda documentation: <span class="exturl"><a class="exturl__link"   target="_blank" rel="noopener" href="https://docs.nvidia.com/cuda/cuda-runtime-api/modules.html#modules" >https://docs.nvidia.com/cuda/cuda-runtime-api/modules.html#modules</a><span class="exturl__icon"><i class="fas fa-external-link-alt"></i></span></span></p>
</blockquote>

        <h2 id="tensorrt-做的工作"   >
          <a href="#tensorrt-做的工作" class="heading-link"><i class="fas fa-link"></i></a><a class="markdownIt-Anchor" href="#tensorrt-做的工作"></a> TensorRT 做的工作</h2>
      
<ul>
<li>构建期
<ul>
<li>模型解析/建立</li>
<li>计算图优化</li>
<li>节点消除</li>
<li>多精度支持</li>
<li>优选kernel/format</li>
<li>导入plugin</li>
<li>显存优化</li>
</ul>
</li>
<li>运行期
<ul>
<li>运行时环境</li>
<li>序列化反序列化</li>
</ul>
</li>
</ul>
<p><a target="_blank" rel="noopener" href="https://www.helloimg.com/image/oGD6Ac">
        <img   class="lazyload lazyload-gif"
          src="/images/loading.svg" data-src="https://www.helloimg.com/images/2023/01/14/oGD6Ac.png"  alt="TensorRT基本流程" border="0" />
      </a></p>

        <h2 id="版本"   >
          <a href="#版本" class="heading-link"><i class="fas fa-link"></i></a><a class="markdownIt-Anchor" href="#版本"></a> 版本</h2>
      
<ol>
<li>GTX1080 / Ubuntu16.04 / cuda10.0 / cudnn7.6.5 / tensorrt7.0.0 / nvinfer7.0.0 / opencv3.3</li>
<li>Yolov5 v4.0</li>
</ol>
<blockquote>
<p><strong>TensorRTX由于基于TensorRT7.0，与最新的8.5.2的API有较大不同</strong><br />
7.0 <span class="exturl"><a class="exturl__link"   target="_blank" rel="noopener" href="https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-700/tensorrt-api/c_api/index.html" >https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-700/tensorrt-api/c_api/index.html</a><span class="exturl__icon"><i class="fas fa-external-link-alt"></i></span></span><br />
latest(8.5.2 now) <span class="exturl"><a class="exturl__link"   target="_blank" rel="noopener" href="https://docs.nvidia.com/deeplearning/tensorrt/api/c_api/index.html" >https://docs.nvidia.com/deeplearning/tensorrt/api/c_api/index.html</a><span class="exturl__icon"><i class="fas fa-external-link-alt"></i></span></span></p>
</blockquote>

        <h2 id="yolov5cpp"   >
          <a href="#yolov5cpp" class="heading-link"><i class="fas fa-link"></i></a><a class="markdownIt-Anchor" href="#yolov5cpp"></a> yolov5.cpp</h2>
      
<span id="more"></span>
<p>首先是宏定义常量：</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">define</span> USE_FP16  <span class="comment">// set USE_INT8 or USE_FP16 or USE_FP32</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> DEVICE 0  <span class="comment">// GPU id</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> NMS_THRESH 0.4</span></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> CONF_THRESH 0.5</span></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> BATCH_SIZE 1</span></span><br></pre></td></tr></table></div></figure>
<p>其中，USE_FP16用于指定量化精度；DEVICE用于指定GPU id，在单显卡状态下默认为0；NMS_THRESH是NMS算法中的筛选阈值；CONF_THRESH是置信度confidence筛选阈值；BATCH_SIZE。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// stuff we know about the network and the input/output blobs</span></span><br><span class="line"><span class="type">static</span> <span class="type">const</span> <span class="type">int</span> INPUT_H = Yolo::INPUT_H;</span><br><span class="line"><span class="type">static</span> <span class="type">const</span> <span class="type">int</span> INPUT_W = Yolo::INPUT_W;</span><br><span class="line"><span class="type">static</span> <span class="type">const</span> <span class="type">int</span> CLASS_NUM = Yolo::CLASS_NUM;</span><br><span class="line"><span class="type">static</span> <span class="type">const</span> <span class="type">int</span> OUTPUT_SIZE = Yolo::MAX_OUTPUT_BBOX_COUNT * <span class="built_in">sizeof</span>(Yolo::Detection) / <span class="built_in">sizeof</span>(<span class="type">float</span>) + <span class="number">1</span>;  <span class="comment">// we assume the yololayer outputs no more than MAX_OUTPUT_BBOX_COUNT boxes that conf &gt;= 0.1</span></span><br><span class="line"><span class="type">const</span> <span class="type">char</span>* INPUT_BLOB_NAME = <span class="string">&quot;data&quot;</span>;</span><br><span class="line"><span class="type">const</span> <span class="type">char</span>* OUTPUT_BLOB_NAME = <span class="string">&quot;prob&quot;</span>;</span><br><span class="line"><span class="type">static</span> Logger gLogger;</span><br></pre></td></tr></table></div></figure>
<p>通过const变量定义了一些常量。其中INPUT_H和INPUT_W指定了输入的尺寸；CLASS_NUM指定了输出标签的种类数量；OUTPUT_SIZE？？。</p>
<p>从main()函数开始看起：</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="type">int</span> <span class="title">main</span><span class="params">(<span class="type">int</span> argc, <span class="type">char</span>** argv)</span> </span>&#123;</span><br><span class="line">    <span class="built_in">cudaSetDevice</span>(DEVICE); <span class="comment">// 指定GPU</span></span><br><span class="line"></span><br><span class="line">    std::string wts_name = <span class="string">&quot;&quot;</span>;</span><br><span class="line">    std::string engine_name = <span class="string">&quot;&quot;</span>;</span><br><span class="line">    <span class="type">float</span> gd = <span class="number">0.0f</span>, gw = <span class="number">0.0f</span>; <span class="comment">// yolov5*.yaml中depth_multiple和width_multiple两个参数</span></span><br><span class="line">    std::string img_dir;</span><br><span class="line">    <span class="comment">// 检查命令行输入参数</span></span><br><span class="line">    <span class="keyword">if</span> (!<span class="built_in">parse_args</span>(argc, argv, wts_name, engine_name, gd, gw, img_dir)) &#123;</span><br><span class="line">        std::cerr &lt;&lt; <span class="string">&quot;arguments not right!&quot;</span> &lt;&lt; std::endl;</span><br><span class="line">        std::cerr &lt;&lt; <span class="string">&quot;./yolov5 -s [.wts] [.engine] [s/m/l/x or c gd gw]  // serialize model to plan file&quot;</span> &lt;&lt; std::endl;</span><br><span class="line">        std::cerr &lt;&lt; <span class="string">&quot;./yolov5 -d [.engine] ../samples  // deserialize plan file and run inference&quot;</span> &lt;&lt; std::endl;</span><br><span class="line">        <span class="keyword">return</span> <span class="number">-1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// create a model using the API directly and serialize it to a stream</span></span><br><span class="line">    <span class="comment">// 通过原生API直接构建模型，然后序列化模型</span></span><br><span class="line">    <span class="keyword">if</span> (!wts_name.<span class="built_in">empty</span>()) &#123;</span><br><span class="line">        IHostMemory* modelStream&#123; <span class="literal">nullptr</span> &#125;;</span><br><span class="line">        <span class="built_in">APIToModel</span>(BATCH_SIZE, &amp;modelStream, gd, gw, wts_name);</span><br><span class="line">        <span class="built_in">assert</span>(modelStream != <span class="literal">nullptr</span>);</span><br><span class="line">        <span class="function">std::ofstream <span class="title">p</span><span class="params">(engine_name, std::ios::binary)</span></span>;</span><br><span class="line">        <span class="keyword">if</span> (!p) &#123;</span><br><span class="line">            std::cerr &lt;&lt; <span class="string">&quot;could not open plan output file&quot;</span> &lt;&lt; std::endl;</span><br><span class="line">            <span class="keyword">return</span> <span class="number">-1</span>;</span><br><span class="line">        &#125;</span><br><span class="line">        p.<span class="built_in">write</span>(<span class="built_in">reinterpret_cast</span>&lt;<span class="type">const</span> <span class="type">char</span>*&gt;(modelStream-&gt;<span class="built_in">data</span>()), modelStream-&gt;<span class="built_in">size</span>());</span><br><span class="line">        modelStream-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">        <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">    &#125;</span><br></pre></td></tr></table></div></figure>
<p>接下来先看看APIToModel()</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * @brief create a model using the API directly</span></span><br><span class="line"><span class="comment">*/</span></span><br><span class="line"><span class="function"><span class="type">void</span> <span class="title">APIToModel</span><span class="params">(<span class="type">unsigned</span> <span class="type">int</span> maxBatchSize, IHostMemory** modelStream, <span class="type">float</span>&amp; gd, <span class="type">float</span>&amp; gw, std::string&amp; wts_name)</span> </span>&#123;</span><br><span class="line">    <span class="comment">// Create builder</span></span><br><span class="line">    <span class="comment">// 建立Builder（引擎构建器）</span></span><br><span class="line">    IBuilder* builder = <span class="built_in">createInferBuilder</span>(gLogger); <span class="comment">// Builder</span></span><br><span class="line">    IBuilderConfig* config = builder-&gt;<span class="built_in">createBuilderConfig</span>(); <span class="comment">// BuilderConfig</span></span><br><span class="line"></span><br><span class="line">    <span class="comment">// Create model to populate the network, then set the outputs and create an engine</span></span><br><span class="line">    <span class="comment">// 建立Engine</span></span><br><span class="line">    ICudaEngine* engine = <span class="built_in">build_engine</span>(maxBatchSize, builder, config, DataType::kFLOAT, gd, gw, wts_name);</span><br><span class="line">    <span class="built_in">assert</span>(engine != <span class="literal">nullptr</span>);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Serialize the engine</span></span><br><span class="line">    <span class="comment">// 序列化</span></span><br><span class="line">    (*modelStream) = engine-&gt;<span class="built_in">serialize</span>();</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Close everything down</span></span><br><span class="line">    engine-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">    builder-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">    config-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">&#125;</span><br></pre></td></tr></table></div></figure>
<p>IBuilder是TensorRT标准类，作用是&quot;Builds an engine from a network definition. &quot;，需要<code>#include &lt;NvInfer.h&gt;</code>。</p>
<p>createInferBuilder是tensorrt7.0的接口，8.5.2中已取消。作用是&quot;Create an instance of an IBuilder class.“，返回的对象是&quot;This class is the logging class for the builder.”。传入参数需要为ILogger对象，是&quot;Application-implemented logging interface for the builder, engine and runtime&quot;，用于生成器、引擎和运行时的应用程序实现的日志记录接口。在TensorRTX中作者继承ILogger类实现了Logger类(在logging.h中)，因此输入的是Logger对象。</p>
<p>builder-&gt;createBuilderConfig()的描述为&quot;Create a builder configuration object.&quot;，即只进行engine的某些配置，不参与网络结构定义。</p>
<p>ICudaEngine是TensorRT运行时推理引擎，描述为&quot;An engine for executing inference on a built network, with functionally unsafe features.&quot;。作者在函数build_engine通过包装原生API实现的网络各层堆叠生成engine。</p>
<p>接下来看build_engine()</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="function">ICudaEngine* <span class="title">build_engine</span><span class="params">(<span class="type">unsigned</span> <span class="type">int</span> maxBatchSize, IBuilder* builder, IBuilderConfig* config, DataType dt, <span class="type">float</span>&amp; gd, <span class="type">float</span>&amp; gw, std::string&amp; wts_name)</span> </span>&#123;</span><br><span class="line">    INetworkDefinition* network = builder-&gt;<span class="built_in">createNetworkV2</span>(<span class="number">0U</span>);</span><br></pre></td></tr></table></div></figure>
<p>输入参数有Builder，BuilderConfig，然后在第一行又通过createNetworkV2()创建了一个network对象。createNetworkV2()的作用是&quot;Create a network definition object.&quot;，即network对象的主要作用为定义网络结构。同时有另一个函数createNetwork()主要用于兼容早期版本的TensorRT。createNetwork()与createNetworkV2()最主要的不同为CreateNetworkV2支持动态形状dynamic shapes 和显式批处理维度explicit batch sizes。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// Create input tensor of shape &#123;3, INPUT_H, INPUT_W&#125; with name INPUT_BLOB_NAME</span></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * @brief Add an input tensor to the network. </span></span><br><span class="line"><span class="comment"> * @param name The name of the tensor.</span></span><br><span class="line"><span class="comment"> * @param type The type of the data held in the tensor.</span></span><br><span class="line"><span class="comment"> * @param dimensions The dimensions of the tensor.</span></span><br><span class="line"><span class="comment">*/</span></span><br><span class="line">ITensor* data = network-&gt;<span class="built_in">addInput</span>(INPUT_BLOB_NAME, dt, Dims3&#123; <span class="number">3</span>, INPUT_H, INPUT_W &#125;);</span><br><span class="line"><span class="built_in">assert</span>(data);</span><br><span class="line"></span><br><span class="line">std::map&lt;std::string, Weights&gt; weightMap = <span class="built_in">loadWeights</span>(wts_name);</span><br><span class="line">Weights emptywts&#123; DataType::kFLOAT, <span class="literal">nullptr</span>, <span class="number">0</span> &#125;;</span><br></pre></td></tr></table></div></figure>
<p>这段主要在定义输入张量大小和加载权重。</p>
<p>network-&gt;addInput()的描述为：</p>
<p>&quot;输入张量的名称name用于查找从网络构建的引擎的缓冲区数组中的索引。尺寸的体积必须小于2^30个元素。对于具有隐式批次维度的网络，此卷包括长度设置为最大批次大小的批次维度。对于具有所有显式维度和通配符维度的网络，体积基于IOptimizationProfile指定的最大值。维度通常为正整数。例外的是，在具有所有显式维度的网络中，-1可以用作在运行时指定维度的通配符。具有此类通配符的输入张量必须在IOptimizationProfiles中具有相应的条目，指示允许的极值，并且输入维度必须由IExecutionContext:setBindingDimensions设置。不同的IExecutionContext实例可以具有不同的维度。只有EngineCapability::kDEFAULT支持通配符维度。它们在安全环境中不受支持。DLA不支持｛C，H，W｝维度中的通配符维度。</p>
<p>张量尺寸的指定与格式无关。例如，如果张量以“NHWC”或矢量化格式格式化，则维度仍按顺序{N，C，H，W}指定。对于具有通道维度的2D图像，最后三个维度总是{C，H，W}。对于具有通道维度的3D图像，最后四个维度总是{C，D，H，W}。&quot;</p>
<p>TensorRTX从.wts文件中加载权重。加载后以map即元组的形式组织。</p>
<p>Weights也是Nvidia的标准类，描述为&quot;An array of weights used as a layer parameter.&quot;，</p>
<p>接下来就是不断堆叠作者通过原生API实现的网络层。</p>
<p>首先是用具体权重实例化每一层对象。这一段类似于yolov5*.yaml</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/* ------ yolov5 backbone------ */</span></span><br><span class="line"><span class="keyword">auto</span> focus0 = <span class="built_in">focus</span>(network, weightMap, *data, <span class="number">3</span>, <span class="built_in">get_width</span>(<span class="number">64</span>, gw), <span class="number">3</span>, <span class="string">&quot;model.0&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> conv1 = <span class="built_in">convBlock</span>(network, weightMap, *focus0-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">128</span>, gw), <span class="number">3</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="string">&quot;model.1&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> bottleneck_CSP2 = <span class="built_in">C3</span>(network, weightMap, *conv1-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">128</span>, gw), <span class="built_in">get_width</span>(<span class="number">128</span>, gw), <span class="built_in">get_depth</span>(<span class="number">3</span>, gd), <span class="literal">true</span>, <span class="number">1</span>, <span class="number">0.5</span>, <span class="string">&quot;model.2&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> conv3 = <span class="built_in">convBlock</span>(network, weightMap, *bottleneck_CSP2-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">256</span>, gw), <span class="number">3</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="string">&quot;model.3&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> bottleneck_csp4 = <span class="built_in">C3</span>(network, weightMap, *conv3-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">256</span>, gw), <span class="built_in">get_width</span>(<span class="number">256</span>, gw), <span class="built_in">get_depth</span>(<span class="number">9</span>, gd), <span class="literal">true</span>, <span class="number">1</span>, <span class="number">0.5</span>, <span class="string">&quot;model.4&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> conv5 = <span class="built_in">convBlock</span>(network, weightMap, *bottleneck_csp4-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), <span class="number">3</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="string">&quot;model.5&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> bottleneck_csp6 = <span class="built_in">C3</span>(network, weightMap, *conv5-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), <span class="built_in">get_depth</span>(<span class="number">9</span>, gd), <span class="literal">true</span>, <span class="number">1</span>, <span class="number">0.5</span>, <span class="string">&quot;model.6&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> conv7 = <span class="built_in">convBlock</span>(network, weightMap, *bottleneck_csp6-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">1024</span>, gw), <span class="number">3</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="string">&quot;model.7&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> spp8 = <span class="built_in">SPP</span>(network, weightMap, *conv7-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">1024</span>, gw), <span class="built_in">get_width</span>(<span class="number">1024</span>, gw), <span class="number">5</span>, <span class="number">9</span>, <span class="number">13</span>, <span class="string">&quot;model.8&quot;</span>);</span><br><span class="line"></span><br><span class="line"><span class="comment">/* ------ yolov5 head ------ */</span></span><br><span class="line"><span class="keyword">auto</span> bottleneck_csp9 = <span class="built_in">C3</span>(network, weightMap, *spp8-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">1024</span>, gw), <span class="built_in">get_width</span>(<span class="number">1024</span>, gw), <span class="built_in">get_depth</span>(<span class="number">3</span>, gd), <span class="literal">false</span>, <span class="number">1</span>, <span class="number">0.5</span>, <span class="string">&quot;model.9&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> conv10 = <span class="built_in">convBlock</span>(network, weightMap, *bottleneck_csp9-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), <span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="string">&quot;model.10&quot;</span>);</span><br><span class="line"></span><br><span class="line"><span class="comment">// reinterpret_cast是c++里面的强制类型转换符，“reinterpret_cast 运算符并不会改变括号中运算对象的值，而是对该对象从位模式上进行重新解释”。malloc是c里面的动态内存分配函数。</span></span><br><span class="line"><span class="type">float</span>* deval = <span class="built_in">reinterpret_cast</span>&lt;<span class="type">float</span>*&gt;(<span class="built_in">malloc</span>(<span class="built_in">sizeof</span>(<span class="type">float</span>) * <span class="built_in">get_width</span>(<span class="number">512</span>, gw) * <span class="number">2</span> * <span class="number">2</span>));</span><br><span class="line"><span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; <span class="built_in">get_width</span>(<span class="number">512</span>, gw) * <span class="number">2</span> * <span class="number">2</span>; i++) &#123;</span><br><span class="line">    deval[i] = <span class="number">1.0</span>; <span class="comment">// 这个deval不知道有什么用</span></span><br><span class="line">&#125;</span><br><span class="line">Weights deconvw</span><br><span class="line">ts11&#123; DataType::kFLOAT, deval, <span class="built_in">get_width</span>(<span class="number">512</span>, gw) * <span class="number">2</span> * <span class="number">2</span> &#125;;</span><br><span class="line">IDeconvolutionLayer* deconv11 = network-&gt;<span class="built_in">addDeconvolutionNd</span>(*conv10-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), DimsHW&#123; <span class="number">2</span>, <span class="number">2</span> &#125;, deconvwts11, emptywts);</span><br><span class="line">deconv11-&gt;<span class="built_in">setStrideNd</span>(DimsHW&#123; <span class="number">2</span>, <span class="number">2</span> &#125;);</span><br><span class="line">deconv11-&gt;<span class="built_in">setNbGroups</span>(<span class="built_in">get_width</span>(<span class="number">512</span>, gw));</span><br><span class="line">weightMap[<span class="string">&quot;deconv11&quot;</span>] = deconvwts11;</span><br><span class="line"></span><br><span class="line">ITensor* inputTensors12[] = &#123; deconv11-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), bottleneck_csp6-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>) &#125;;</span><br><span class="line"><span class="keyword">auto</span> cat12 = network-&gt;<span class="built_in">addConcatenation</span>(inputTensors12, <span class="number">2</span>);</span><br><span class="line"><span class="keyword">auto</span> bottleneck_csp13 = <span class="built_in">C3</span>(network, weightMap, *cat12-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">1024</span>, gw), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), <span class="built_in">get_depth</span>(<span class="number">3</span>, gd), <span class="literal">false</span>, <span class="number">1</span>, <span class="number">0.5</span>, <span class="string">&quot;model.13&quot;</span>);</span><br><span class="line"><span class="keyword">auto</span> conv14 = <span class="built_in">convBlock</span>(network, weightMap, *bottleneck_csp13-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">256</span>, gw), <span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="string">&quot;model.14&quot;</span>);</span><br><span class="line"></span><br><span class="line">Weights deconvwts15&#123; DataType::kFLOAT, deval, <span class="built_in">get_width</span>(<span class="number">256</span>, gw) * <span class="number">2</span> * <span class="number">2</span> &#125;;</span><br><span class="line">IDeconvolutionLayer* deconv15 = network-&gt;<span class="built_in">addDeconvolutionNd</span>(*conv14-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">256</span>, gw), DimsHW&#123; <span class="number">2</span>, <span class="number">2</span> &#125;, deconvwts15, emptywts);</span><br><span class="line">deconv15-&gt;<span class="built_in">setStrideNd</span>(DimsHW&#123; <span class="number">2</span>, <span class="number">2</span> &#125;);</span><br><span class="line">deconv15-&gt;<span class="built_in">setNbGroups</span>(<span class="built_in">get_width</span>(<span class="number">256</span>, gw));</span><br><span class="line">ITensor* inputTensors16[] = &#123; deconv15-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), bottleneck_csp4-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>) &#125;;</span><br><span class="line"><span class="keyword">auto</span> cat16 = network-&gt;<span class="built_in">addConcatenation</span>(inputTensors16, <span class="number">2</span>);</span><br><span class="line"></span><br><span class="line"><span class="keyword">auto</span> bottleneck_csp17 = <span class="built_in">C3</span>(network, weightMap, *cat16-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), <span class="built_in">get_width</span>(<span class="number">256</span>, gw), <span class="built_in">get_depth</span>(<span class="number">3</span>, gd), <span class="literal">false</span>, <span class="number">1</span>, <span class="number">0.5</span>, <span class="string">&quot;model.17&quot;</span>);</span><br><span class="line"></span><br><span class="line"><span class="comment">// yolo layer 0</span></span><br><span class="line">IConvolutionLayer* det0 = network-&gt;<span class="built_in">addConvolutionNd</span>(*bottleneck_csp17-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="number">3</span> * (Yolo::CLASS_NUM + <span class="number">5</span>), DimsHW&#123; <span class="number">1</span>, <span class="number">1</span> &#125;, weightMap[<span class="string">&quot;model.24.m.0.weight&quot;</span>], weightMap[<span class="string">&quot;model.24.m.0.bias&quot;</span>]);</span><br><span class="line"><span class="keyword">auto</span> conv18 = <span class="built_in">convBlock</span>(network, weightMap, *bottleneck_csp17-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">256</span>, gw), <span class="number">3</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="string">&quot;model.18&quot;</span>);</span><br><span class="line">ITensor* inputTensors19[] = &#123; conv18-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), conv14-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>) &#125;;</span><br><span class="line"><span class="keyword">auto</span> cat19 = network-&gt;<span class="built_in">addConcatenation</span>(inputTensors19, <span class="number">2</span>);</span><br><span class="line"><span class="keyword">auto</span> bottleneck_csp20 = <span class="built_in">C3</span>(network, weightMap, *cat19-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), <span class="built_in">get_depth</span>(<span class="number">3</span>, gd), <span class="literal">false</span>, <span class="number">1</span>, <span class="number">0.5</span>, <span class="string">&quot;model.20&quot;</span>);</span><br><span class="line"><span class="comment">//yolo layer 1</span></span><br><span class="line">IConvolutionLayer* det1 = network-&gt;<span class="built_in">addConvolutionNd</span>(*bottleneck_csp20-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="number">3</span> * (Yolo::CLASS_NUM + <span class="number">5</span>), DimsHW&#123; <span class="number">1</span>, <span class="number">1</span> &#125;, weightMap[<span class="string">&quot;model.24.m.1.weight&quot;</span>], weightMap[<span class="string">&quot;model.24.m.1.bias&quot;</span>]);</span><br><span class="line"><span class="keyword">auto</span> conv21 = <span class="built_in">convBlock</span>(network, weightMap, *bottleneck_csp20-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">512</span>, gw), <span class="number">3</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="string">&quot;model.21&quot;</span>);</span><br><span class="line">ITensor* inputTensors22[] = &#123; conv21-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), conv10-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>) &#125;;</span><br><span class="line"><span class="keyword">auto</span> cat22 = network-&gt;<span class="built_in">addConcatenation</span>(inputTensors22, <span class="number">2</span>);</span><br><span class="line"><span class="keyword">auto</span> bottleneck_csp23 = <span class="built_in">C3</span>(network, weightMap, *cat22-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="built_in">get_width</span>(<span class="number">1024</span>, gw), <span class="built_in">get_width</span>(<span class="number">1024</span>, gw), <span class="built_in">get_depth</span>(<span class="number">3</span>, gd), <span class="literal">false</span>, <span class="number">1</span>, <span class="number">0.5</span>, <span class="string">&quot;model.23&quot;</span>);</span><br><span class="line">IConvolutionLayer* det2 = network-&gt;<span class="built_in">addConvolutionNd</span>(*bottleneck_csp23-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), <span class="number">3</span> * (Yolo::CLASS_NUM + <span class="number">5</span>), DimsHW&#123; <span class="number">1</span>, <span class="number">1</span> &#125;, weightMap[<span class="string">&quot;model.24.m.2.weight&quot;</span>], weightMap[<span class="string">&quot;model.24.m.2.bias&quot;</span>]);</span><br><span class="line"></span><br><span class="line"><span class="keyword">auto</span> yolo = <span class="built_in">addYoLoLayer</span>(network, weightMap, det0, det1, det2);</span><br><span class="line">yolo-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>)-&gt;<span class="built_in">setName</span>(OUTPUT_BLOB_NAME);</span><br><span class="line">network-&gt;<span class="built_in">markOutput</span>(*yolo-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>));</span><br></pre></td></tr></table></div></figure>
<p>这是Yolov5 v4.0的网络结构</p>
<figure class="highlight yaml"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># parameters</span></span><br><span class="line"><span class="attr">nc:</span> <span class="number">80</span>  <span class="comment"># number of classes</span></span><br><span class="line"><span class="attr">depth_multiple:</span> <span class="number">0.33</span>  <span class="comment"># model depth multiple</span></span><br><span class="line"><span class="attr">width_multiple:</span> <span class="number">0.50</span>  <span class="comment"># layer channel multiple</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># anchors</span></span><br><span class="line"><span class="attr">anchors:</span></span><br><span class="line">  <span class="bullet">-</span> [<span class="number">10</span>,<span class="number">13</span>, <span class="number">16</span>,<span class="number">30</span>, <span class="number">33</span>,<span class="number">23</span>]  <span class="comment"># P3/8</span></span><br><span class="line">  <span class="bullet">-</span> [<span class="number">30</span>,<span class="number">61</span>, <span class="number">62</span>,<span class="number">45</span>, <span class="number">59</span>,<span class="number">119</span>]  <span class="comment"># P4/16</span></span><br><span class="line">  <span class="bullet">-</span> [<span class="number">116</span>,<span class="number">90</span>, <span class="number">156</span>,<span class="number">198</span>, <span class="number">373</span>,<span class="number">326</span>]  <span class="comment"># P5/32</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># YOLOv5 backbone</span></span><br><span class="line"><span class="attr">backbone:</span></span><br><span class="line">  <span class="comment"># [from, number, module, args]</span></span><br><span class="line">  [[<span class="number">-1</span>, <span class="number">1</span>, <span class="string">Focus</span>, [<span class="number">64</span>, <span class="number">3</span>]],  <span class="comment"># 0-P1/2</span></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">Conv</span>, [<span class="number">128</span>, <span class="number">3</span>, <span class="number">2</span>]],  <span class="comment"># 1-P2/4</span></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">3</span>, <span class="string">C3</span>, [<span class="number">128</span>]],</span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">Conv</span>, [<span class="number">256</span>, <span class="number">3</span>, <span class="number">2</span>]],  <span class="comment"># 3-P3/8</span></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">9</span>, <span class="string">C3</span>, [<span class="number">256</span>]],</span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">Conv</span>, [<span class="number">512</span>, <span class="number">3</span>, <span class="number">2</span>]],  <span class="comment"># 5-P4/16</span></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">9</span>, <span class="string">C3</span>, [<span class="number">512</span>]],</span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">Conv</span>, [<span class="number">1024</span>, <span class="number">3</span>, <span class="number">2</span>]],  <span class="comment"># 7-P5/32</span></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">SPP</span>, [<span class="number">1024</span>, [<span class="number">5</span>, <span class="number">9</span>, <span class="number">13</span>]]],</span><br><span class="line">   [<span class="number">-1</span>, <span class="number">3</span>, <span class="string">C3</span>, [<span class="number">1024</span>, <span class="literal">False</span>]],  <span class="comment"># 9</span></span><br><span class="line">  ]</span><br><span class="line"></span><br><span class="line"><span class="comment"># YOLOv5 head</span></span><br><span class="line"><span class="attr">head:</span></span><br><span class="line">  [[<span class="number">-1</span>, <span class="number">1</span>, <span class="string">Conv</span>, [<span class="number">512</span>, <span class="number">1</span>, <span class="number">1</span>]],</span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">nn.Upsample</span>, [<span class="string">None</span>, <span class="number">2</span>, <span class="string">&#x27;nearest&#x27;</span>]],</span><br><span class="line">   [[<span class="number">-1</span>, <span class="number">6</span>], <span class="number">1</span>, <span class="string">Concat</span>, [<span class="number">1</span>]],  <span class="comment"># cat backbone P4</span></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">3</span>, <span class="string">C3</span>, [<span class="number">512</span>, <span class="literal">False</span>]],  <span class="comment"># 13</span></span><br><span class="line"></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">Conv</span>, [<span class="number">256</span>, <span class="number">1</span>, <span class="number">1</span>]],</span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">nn.Upsample</span>, [<span class="string">None</span>, <span class="number">2</span>, <span class="string">&#x27;nearest&#x27;</span>]],</span><br><span class="line">   [[<span class="number">-1</span>, <span class="number">4</span>], <span class="number">1</span>, <span class="string">Concat</span>, [<span class="number">1</span>]],  <span class="comment"># cat backbone P3</span></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">3</span>, <span class="string">C3</span>, [<span class="number">256</span>, <span class="literal">False</span>]],  <span class="comment"># 17 (P3/8-small)</span></span><br><span class="line"></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">Conv</span>, [<span class="number">256</span>, <span class="number">3</span>, <span class="number">2</span>]],</span><br><span class="line">   [[<span class="number">-1</span>, <span class="number">14</span>], <span class="number">1</span>, <span class="string">Concat</span>, [<span class="number">1</span>]],  <span class="comment"># cat head P4</span></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">3</span>, <span class="string">C3</span>, [<span class="number">512</span>, <span class="literal">False</span>]],  <span class="comment"># 20 (P4/16-medium)</span></span><br><span class="line"></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">1</span>, <span class="string">Conv</span>, [<span class="number">512</span>, <span class="number">3</span>, <span class="number">2</span>]],</span><br><span class="line">   [[<span class="number">-1</span>, <span class="number">10</span>], <span class="number">1</span>, <span class="string">Concat</span>, [<span class="number">1</span>]],  <span class="comment"># cat head P5</span></span><br><span class="line">   [<span class="number">-1</span>, <span class="number">3</span>, <span class="string">C3</span>, [<span class="number">1024</span>, <span class="literal">False</span>]],  <span class="comment"># 23 (P5/32-large)</span></span><br><span class="line"></span><br><span class="line">   [[<span class="number">17</span>, <span class="number">20</span>, <span class="number">23</span>], <span class="number">1</span>, <span class="string">Detect</span>, [<span class="string">nc</span>, <span class="string">anchors</span>]],  <span class="comment"># Detect(P3, P4, P5)</span></span><br><span class="line">  ]</span><br></pre></td></tr></table></div></figure>
<p>其中以convBlock为例，对比yolov5 pytorch和TensorRTX中基于原生API的实现。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="function">ILayer* <span class="title">convBlock</span><span class="params">(INetworkDefinition *network, std::map&lt;std::string, Weights&gt;&amp; weightMap, ITensor&amp; input, <span class="type">int</span> outch, <span class="type">int</span> ksize, <span class="type">int</span> s, <span class="type">int</span> g, std::string lname)</span> </span>&#123;</span><br><span class="line">    Weights emptywts&#123; DataType::kFLOAT, <span class="literal">nullptr</span>, <span class="number">0</span> &#125;;</span><br><span class="line">    <span class="type">int</span> p = ksize / <span class="number">2</span>;</span><br><span class="line">    IConvolutionLayer* conv1 = network-&gt;<span class="built_in">addConvolutionNd</span>(input, outch, DimsHW&#123; ksize, ksize &#125;, weightMap[lname + <span class="string">&quot;.conv.weight&quot;</span>], emptywts);</span><br><span class="line">    <span class="built_in">assert</span>(conv1);</span><br><span class="line">    conv1-&gt;<span class="built_in">setStrideNd</span>(DimsHW&#123; s, s &#125;);</span><br><span class="line">    conv1-&gt;<span class="built_in">setPaddingNd</span>(DimsHW&#123; p, p &#125;);</span><br><span class="line">    conv1-&gt;<span class="built_in">setNbGroups</span>(g);</span><br><span class="line">    IScaleLayer* bn1 = <span class="built_in">addBatchNorm2d</span>(network, weightMap, *conv1-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), lname + <span class="string">&quot;.bn&quot;</span>, <span class="number">1e-3</span>);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// silu = x * sigmoid</span></span><br><span class="line">    <span class="keyword">auto</span> sig = network-&gt;<span class="built_in">addActivation</span>(*bn1-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), ActivationType::kSIGMOID);</span><br><span class="line">    <span class="built_in">assert</span>(sig);</span><br><span class="line">    <span class="keyword">auto</span> ew = network-&gt;<span class="built_in">addElementWise</span>(*bn1-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), *sig-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), ElementWiseOperation::kPROD);</span><br><span class="line">    <span class="built_in">assert</span>(ew);</span><br><span class="line">    <span class="keyword">return</span> ew;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></div></figure>
<figure class="highlight python"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># Yolov5 v4.0</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Conv</span>(nn.Module):</span><br><span class="line">    <span class="comment"># Standard convolution</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, c1, c2, k=<span class="number">1</span>, s=<span class="number">1</span>, p=<span class="literal">None</span>, g=<span class="number">1</span>, act=<span class="literal">True</span></span>):  <span class="comment"># ch_in, ch_out, kernel, stride, padding, groups</span></span><br><span class="line">        <span class="built_in">super</span>(Conv, self).__init__()</span><br><span class="line">        self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=<span class="literal">False</span>)</span><br><span class="line">        self.bn = nn.BatchNorm2d(c2)</span><br><span class="line">        self.act = nn.SiLU() <span class="keyword">if</span> act <span class="keyword">is</span> <span class="literal">True</span> <span class="keyword">else</span> (act <span class="keyword">if</span> <span class="built_in">isinstance</span>(act, nn.Module) <span class="keyword">else</span> nn.Identity())</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> self.act(self.bn(self.conv(x)))</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">fuseforward</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> self.act(self.conv(x))</span><br></pre></td></tr></table></div></figure>
<p>对比一下，可以看到并不能直接从pytorch翻译成TensorRT，需要添加很多细节。这应该是由于pytorch框架内部隐含实现了很多细节。但是只要网络是基于pytorch实现的，都可以参考别人的翻译来实现，因为在翻译时只需要以pytorch API为单位进行翻译即可。</p>
<p>注意到在common.hpp和yololayer.cu中作者自行实现了一个plugin，</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// common.hpp</span></span><br><span class="line"><span class="function">IPluginV2Layer* <span class="title">addYoLoLayer</span><span class="params">(INetworkDefinition *network, std::map&lt;std::string, Weights&gt;&amp; weightMap, IConvolutionLayer* det0, IConvolutionLayer* det1, IConvolutionLayer* det2)</span></span></span><br><span class="line"><span class="function"></span>&#123;</span><br><span class="line">    <span class="keyword">auto</span> creator = <span class="built_in">getPluginRegistry</span>()-&gt;<span class="built_in">getPluginCreator</span>(<span class="string">&quot;YoloLayer_TRT&quot;</span>, <span class="string">&quot;1&quot;</span>);</span><br><span class="line">    std::vector&lt;<span class="type">float</span>&gt; anchors_yolo = <span class="built_in">getAnchors</span>(weightMap);</span><br><span class="line">    PluginField pluginMultidata[<span class="number">4</span>];</span><br><span class="line">    <span class="type">int</span> NetData[<span class="number">4</span>];</span><br><span class="line">    NetData[<span class="number">0</span>] = Yolo::CLASS_NUM;</span><br><span class="line">    NetData[<span class="number">1</span>] = Yolo::INPUT_W;</span><br><span class="line">    NetData[<span class="number">2</span>] = Yolo::INPUT_H;</span><br><span class="line">    NetData[<span class="number">3</span>] = Yolo::MAX_OUTPUT_BBOX_COUNT;</span><br><span class="line">    pluginMultidata[<span class="number">0</span>].data = NetData;</span><br><span class="line">    pluginMultidata[<span class="number">0</span>].length = <span class="number">3</span>;</span><br><span class="line">    pluginMultidata[<span class="number">0</span>].name = <span class="string">&quot;netdata&quot;</span>;</span><br><span class="line">    pluginMultidata[<span class="number">0</span>].type = PluginFieldType::kFLOAT32;</span><br><span class="line">    <span class="type">int</span> scale[<span class="number">3</span>] = &#123; <span class="number">8</span>, <span class="number">16</span>, <span class="number">32</span> &#125;;</span><br><span class="line">    <span class="type">int</span> plugindata[<span class="number">3</span>][<span class="number">8</span>];</span><br><span class="line">    std::string names[<span class="number">3</span>];</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">1</span>; k &lt; <span class="number">4</span>; k++)</span><br><span class="line">    &#123;</span><br><span class="line">        plugindata[k - <span class="number">1</span>][<span class="number">0</span>] = Yolo::INPUT_W / scale[k - <span class="number">1</span>];</span><br><span class="line">        plugindata[k - <span class="number">1</span>][<span class="number">1</span>] = Yolo::INPUT_H / scale[k - <span class="number">1</span>];</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">2</span>; i &lt; <span class="number">8</span>; i++)</span><br><span class="line">        &#123;</span><br><span class="line">            plugindata[k - <span class="number">1</span>][i] = <span class="built_in">int</span>(anchors_yolo[(k - <span class="number">1</span>) * <span class="number">6</span> + i - <span class="number">2</span>]);</span><br><span class="line">        &#125;</span><br><span class="line">        pluginMultidata[k].data = plugindata[k - <span class="number">1</span>];</span><br><span class="line">        pluginMultidata[k].length = <span class="number">8</span>;</span><br><span class="line">        names[k - <span class="number">1</span>] = <span class="string">&quot;yolodata&quot;</span> + std::<span class="built_in">to_string</span>(k);</span><br><span class="line">        pluginMultidata[k].name = names[k - <span class="number">1</span>].<span class="built_in">c_str</span>();</span><br><span class="line">        pluginMultidata[k].type = PluginFieldType::kFLOAT32;</span><br><span class="line">    &#125;</span><br><span class="line">    PluginFieldCollection pluginData;</span><br><span class="line">    pluginData.nbFields = <span class="number">4</span>;</span><br><span class="line">    pluginData.fields = pluginMultidata;</span><br><span class="line">    IPluginV2 *pluginObj = creator-&gt;<span class="built_in">createPlugin</span>(<span class="string">&quot;yololayer&quot;</span>, &amp;pluginData);</span><br><span class="line">    ITensor* inputTensors_yolo[] = &#123; det2-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), det1-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>), det0-&gt;<span class="built_in">getOutput</span>(<span class="number">0</span>) &#125;;</span><br><span class="line">    <span class="keyword">auto</span> yolo = network-&gt;<span class="built_in">addPluginV2</span>(inputTensors_yolo, <span class="number">3</span>, *pluginObj);</span><br><span class="line">    <span class="keyword">return</span> yolo;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// yololayer.h</span></span><br><span class="line"><span class="keyword">namespace</span> Yolo</span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">static</span> <span class="keyword">constexpr</span> <span class="type">int</span> CHECK_COUNT = <span class="number">3</span>;</span><br><span class="line">    <span class="type">static</span> <span class="keyword">constexpr</span> <span class="type">float</span> IGNORE_THRESH = <span class="number">0.1f</span>;</span><br><span class="line">    <span class="keyword">struct</span> <span class="title class_">YoloKernel</span></span><br><span class="line">    &#123;</span><br><span class="line">        <span class="type">int</span> width;</span><br><span class="line">        <span class="type">int</span> height;</span><br><span class="line">        <span class="type">float</span> anchors[CHECK_COUNT * <span class="number">2</span>];</span><br><span class="line">    &#125;;</span><br><span class="line">    <span class="type">static</span> <span class="keyword">constexpr</span> <span class="type">int</span> MAX_OUTPUT_BBOX_COUNT = <span class="number">1000</span>;</span><br><span class="line">    <span class="type">static</span> <span class="keyword">constexpr</span> <span class="type">int</span> CLASS_NUM = <span class="number">80</span>;</span><br><span class="line">    <span class="type">static</span> <span class="keyword">constexpr</span> <span class="type">int</span> INPUT_H = <span class="number">608</span>;</span><br><span class="line">    <span class="type">static</span> <span class="keyword">constexpr</span> <span class="type">int</span> INPUT_W = <span class="number">608</span>;</span><br><span class="line"></span><br><span class="line">    <span class="type">static</span> <span class="keyword">constexpr</span> <span class="type">int</span> LOCATIONS = <span class="number">4</span>;</span><br><span class="line">    <span class="keyword">struct</span> <span class="title class_">alignas</span>(<span class="type">float</span>) Detection &#123;</span><br><span class="line">        <span class="comment">//center_x center_y w h</span></span><br><span class="line">        <span class="type">float</span> bbox[LOCATIONS];</span><br><span class="line">        <span class="type">float</span> conf;  <span class="comment">// bbox_conf * cls_conf</span></span><br><span class="line">        <span class="type">float</span> class_id;</span><br><span class="line">    &#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="keyword">namespace</span> nvinfer1</span><br><span class="line">&#123;</span><br><span class="line">    <span class="keyword">class</span> <span class="title class_">YoloLayerPlugin</span> : <span class="keyword">public</span> IPluginV2IOExt</span><br><span class="line">    &#123;</span><br><span class="line">    <span class="keyword">public</span>:</span><br><span class="line">        <span class="built_in">YoloLayerPlugin</span>(<span class="type">int</span> classCount, <span class="type">int</span> netWidth, <span class="type">int</span> netHeight, <span class="type">int</span> maxOut, <span class="type">const</span> std::vector&lt;Yolo::YoloKernel&gt;&amp; vYoloKernel);</span><br><span class="line">        <span class="built_in">YoloLayerPlugin</span>(<span class="type">const</span> <span class="type">void</span>* data, <span class="type">size_t</span> length);</span><br><span class="line">        ~<span class="built_in">YoloLayerPlugin</span>();</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">int</span> <span class="title">getNbOutputs</span><span class="params">()</span> <span class="type">const</span> <span class="keyword">override</span></span></span><br><span class="line"><span class="function">        </span>&#123;</span><br><span class="line">            <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="function">Dims <span class="title">getOutputDimensions</span><span class="params">(<span class="type">int</span> index, <span class="type">const</span> Dims* inputs, <span class="type">int</span> nbInputDims)</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">int</span> <span class="title">initialize</span><span class="params">()</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="keyword">virtual</span> <span class="type">void</span> <span class="title">terminate</span><span class="params">()</span> <span class="keyword">override</span> </span>&#123;&#125;;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="keyword">virtual</span> <span class="type">size_t</span> <span class="title">getWorkspaceSize</span><span class="params">(<span class="type">int</span> maxBatchSize)</span> <span class="type">const</span> <span class="keyword">override</span> </span>&#123; <span class="keyword">return</span> <span class="number">0</span>; &#125;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="keyword">virtual</span> <span class="type">int</span> <span class="title">enqueue</span><span class="params">(<span class="type">int</span> batchSize, <span class="type">const</span> <span class="type">void</span>*<span class="type">const</span> * inputs, <span class="type">void</span>** outputs, <span class="type">void</span>* workspace, cudaStream_t stream)</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="keyword">virtual</span> <span class="type">size_t</span> <span class="title">getSerializationSize</span><span class="params">()</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="keyword">virtual</span> <span class="type">void</span> <span class="title">serialize</span><span class="params">(<span class="type">void</span>* buffer)</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">bool</span> <span class="title">supportsFormatCombination</span><span class="params">(<span class="type">int</span> pos, <span class="type">const</span> PluginTensorDesc* inOut, <span class="type">int</span> nbInputs, <span class="type">int</span> nbOutputs)</span> <span class="type">const</span> <span class="keyword">override</span> </span>&#123;</span><br><span class="line">            <span class="keyword">return</span> inOut[pos].format == TensorFormat::kLINEAR &amp;&amp; inOut[pos].type == DataType::kFLOAT;</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">const</span> <span class="type">char</span>* <span class="title">getPluginType</span><span class="params">()</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">const</span> <span class="type">char</span>* <span class="title">getPluginVersion</span><span class="params">()</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">void</span> <span class="title">destroy</span><span class="params">()</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function">IPluginV2IOExt* <span class="title">clone</span><span class="params">()</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">void</span> <span class="title">setPluginNamespace</span><span class="params">(<span class="type">const</span> <span class="type">char</span>* pluginNamespace)</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">const</span> <span class="type">char</span>* <span class="title">getPluginNamespace</span><span class="params">()</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function">DataType <span class="title">getOutputDataType</span><span class="params">(<span class="type">int</span> index, <span class="type">const</span> nvinfer1::DataType* inputTypes, <span class="type">int</span> nbInputs)</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">bool</span> <span class="title">isOutputBroadcastAcrossBatch</span><span class="params">(<span class="type">int</span> outputIndex, <span class="type">const</span> <span class="type">bool</span>* inputIsBroadcasted, <span class="type">int</span> nbInputs)</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">bool</span> <span class="title">canBroadcastInputAcrossBatch</span><span class="params">(<span class="type">int</span> inputIndex)</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">void</span> <span class="title">attachToContext</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">            cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator)</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">void</span> <span class="title">configurePlugin</span><span class="params">(<span class="type">const</span> PluginTensorDesc* in, <span class="type">int</span> nbInput, <span class="type">const</span> PluginTensorDesc* out, <span class="type">int</span> nbOutput)</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">void</span> <span class="title">detachFromContext</span><span class="params">()</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span>:</span><br><span class="line">        <span class="function"><span class="type">void</span> <span class="title">forwardGpu</span><span class="params">(<span class="type">const</span> <span class="type">float</span> *<span class="type">const</span> * inputs, <span class="type">float</span> * output, cudaStream_t stream, <span class="type">int</span> batchSize = <span class="number">1</span>)</span></span>;</span><br><span class="line">        <span class="type">int</span> mThreadCount = <span class="number">256</span>;</span><br><span class="line">        <span class="type">const</span> <span class="type">char</span>* mPluginNamespace;</span><br><span class="line">        <span class="type">int</span> mKernelCount;</span><br><span class="line">        <span class="type">int</span> mClassCount;</span><br><span class="line">        <span class="type">int</span> mYoloV5NetWidth;</span><br><span class="line">        <span class="type">int</span> mYoloV5NetHeight;</span><br><span class="line">        <span class="type">int</span> mMaxOutObject;</span><br><span class="line">        std::vector&lt;Yolo::YoloKernel&gt; mYoloKernel;</span><br><span class="line">        <span class="type">void</span>** mAnchor;</span><br><span class="line">    &#125;;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">class</span> <span class="title class_">YoloPluginCreator</span> : <span class="keyword">public</span> IPluginCreator</span><br><span class="line">    &#123;</span><br><span class="line">    <span class="keyword">public</span>:</span><br><span class="line">        <span class="built_in">YoloPluginCreator</span>();</span><br><span class="line"></span><br><span class="line">        ~<span class="built_in">YoloPluginCreator</span>() <span class="keyword">override</span> = <span class="keyword">default</span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">const</span> <span class="type">char</span>* <span class="title">getPluginName</span><span class="params">()</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">const</span> <span class="type">char</span>* <span class="title">getPluginVersion</span><span class="params">()</span> <span class="type">const</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">const</span> PluginFieldCollection* <span class="title">getFieldNames</span><span class="params">()</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function">IPluginV2IOExt* <span class="title">createPlugin</span><span class="params">(<span class="type">const</span> <span class="type">char</span>* name, <span class="type">const</span> PluginFieldCollection* fc)</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function">IPluginV2IOExt* <span class="title">deserializePlugin</span><span class="params">(<span class="type">const</span> <span class="type">char</span>* name, <span class="type">const</span> <span class="type">void</span>* serialData, <span class="type">size_t</span> serialLength)</span> <span class="keyword">override</span></span>;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">void</span> <span class="title">setPluginNamespace</span><span class="params">(<span class="type">const</span> <span class="type">char</span>* libNamespace)</span> <span class="keyword">override</span></span></span><br><span class="line"><span class="function">        </span>&#123;</span><br><span class="line">            mNamespace = libNamespace;</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="type">const</span> <span class="type">char</span>* <span class="title">getPluginNamespace</span><span class="params">()</span> <span class="type">const</span> <span class="keyword">override</span></span></span><br><span class="line"><span class="function">        </span>&#123;</span><br><span class="line">            <span class="keyword">return</span> mNamespace.<span class="built_in">c_str</span>();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span>:</span><br><span class="line">        std::string mNamespace;</span><br><span class="line">        <span class="type">static</span> PluginFieldCollection mFC;</span><br><span class="line">        <span class="type">static</span> std::vector&lt;PluginField&gt; mPluginAttributes;</span><br><span class="line">    &#125;;</span><br><span class="line">    <span class="built_in">REGISTER_TENSORRT_PLUGIN</span>(YoloPluginCreator);</span><br><span class="line">&#125;;</span><br></pre></td></tr></table></div></figure>
<p>可以看到plugin是cuda编程实现的，具体是那一部分？估计大概率是多重检测头。<span class="exturl"><a class="exturl__link"   target="_blank" rel="noopener" href="http://xn--yolov5common-kw8up86cu89c7z9a.py" >对比回yolov5的common.py</a><span class="exturl__icon"><i class="fas fa-external-link-alt"></i></span></span></p>
<figure class="highlight python"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">DetectMultiBackend</span>(nn.Module):</span><br><span class="line">    <span class="comment"># YOLOv5 MultiBackend class for python inference on various backends</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, weights=<span class="string">&#x27;yolov5s.pt&#x27;</span>, device=torch.device(<span class="params"><span class="string">&#x27;cpu&#x27;</span></span>), dnn=<span class="literal">False</span>, data=<span class="literal">None</span>, fp16=<span class="literal">False</span>, fuse=<span class="literal">True</span></span>):</span><br><span class="line">        <span class="comment"># Usage:</span></span><br><span class="line">        <span class="comment">#   PyTorch:              weights = *.pt</span></span><br><span class="line">        <span class="comment">#   TorchScript:                    *.torchscript</span></span><br><span class="line">        <span class="comment">#   ONNX Runtime:                   *.onnx</span></span><br><span class="line">        <span class="comment">#   ONNX OpenCV DNN:                *.onnx --dnn</span></span><br><span class="line">        <span class="comment">#   OpenVINO:                       *.xml</span></span><br><span class="line">        <span class="comment">#   CoreML:                         *.mlmodel</span></span><br><span class="line">        <span class="comment">#   TensorRT:                       *.engine</span></span><br><span class="line">        <span class="comment">#   TensorFlow SavedModel:          *_saved_model</span></span><br><span class="line">        <span class="comment">#   TensorFlow GraphDef:            *.pb</span></span><br><span class="line">        <span class="comment">#   TensorFlow Lite:                *.tflite</span></span><br><span class="line">        <span class="comment">#   TensorFlow Edge TPU:            *_edgetpu.tflite</span></span><br><span class="line">        <span class="comment">#   PaddlePaddle:                   *_paddle_model</span></span><br><span class="line">        <span class="keyword">from</span> models.experimental <span class="keyword">import</span> attempt_download, attempt_load  <span class="comment"># scoped to avoid circular import</span></span><br><span class="line"></span><br><span class="line">        <span class="built_in">super</span>().__init__()</span><br><span class="line">        w = <span class="built_in">str</span>(weights[<span class="number">0</span>] <span class="keyword">if</span> <span class="built_in">isinstance</span>(weights, <span class="built_in">list</span>) <span class="keyword">else</span> weights)</span><br><span class="line">        pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle, triton = self._model_type(w)</span><br><span class="line">        fp16 &amp;= pt <span class="keyword">or</span> jit <span class="keyword">or</span> onnx <span class="keyword">or</span> engine  <span class="comment"># FP16</span></span><br><span class="line">        nhwc = coreml <span class="keyword">or</span> saved_model <span class="keyword">or</span> pb <span class="keyword">or</span> tflite <span class="keyword">or</span> edgetpu  <span class="comment"># BHWC formats (vs torch BCWH)</span></span><br><span class="line">        stride = <span class="number">32</span>  <span class="comment"># default stride</span></span><br><span class="line">        cuda = torch.cuda.is_available() <span class="keyword">and</span> device.<span class="built_in">type</span> != <span class="string">&#x27;cpu&#x27;</span>  <span class="comment"># use CUDA</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> (pt <span class="keyword">or</span> triton):</span><br><span class="line">            w = attempt_download(w)  <span class="comment"># download if not local</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> pt:  <span class="comment"># PyTorch</span></span><br><span class="line">            model = attempt_load(weights <span class="keyword">if</span> <span class="built_in">isinstance</span>(weights, <span class="built_in">list</span>) <span class="keyword">else</span> w, device=device, inplace=<span class="literal">True</span>, fuse=fuse)</span><br><span class="line">            stride = <span class="built_in">max</span>(<span class="built_in">int</span>(model.stride.<span class="built_in">max</span>()), <span class="number">32</span>)  <span class="comment"># model stride</span></span><br><span class="line">            names = model.module.names <span class="keyword">if</span> <span class="built_in">hasattr</span>(model, <span class="string">&#x27;module&#x27;</span>) <span class="keyword">else</span> model.names  <span class="comment"># get class names</span></span><br><span class="line">            model.half() <span class="keyword">if</span> fp16 <span class="keyword">else</span> model.<span class="built_in">float</span>()</span><br><span class="line">            self.model = model  <span class="comment"># explicitly assign for to(), cpu(), cuda(), half()</span></span><br><span class="line">        ......</span><br><span class="line">        <span class="keyword">elif</span> engine:  <span class="comment"># TensorRT</span></span><br><span class="line">            LOGGER.info(<span class="string">f&#x27;Loading <span class="subst">&#123;w&#125;</span> for TensorRT inference...&#x27;</span>)</span><br><span class="line">            <span class="keyword">import</span> tensorrt <span class="keyword">as</span> trt  <span class="comment"># https://developer.nvidia.com/nvidia-tensorrt-download</span></span><br><span class="line">            check_version(trt.__version__, <span class="string">&#x27;7.0.0&#x27;</span>, hard=<span class="literal">True</span>)  <span class="comment"># require tensorrt&gt;=7.0.0</span></span><br><span class="line">            <span class="keyword">if</span> device.<span class="built_in">type</span> == <span class="string">&#x27;cpu&#x27;</span>:</span><br><span class="line">                device = torch.device(<span class="string">&#x27;cuda:0&#x27;</span>)</span><br><span class="line">            Binding = namedtuple(<span class="string">&#x27;Binding&#x27;</span>, (<span class="string">&#x27;name&#x27;</span>, <span class="string">&#x27;dtype&#x27;</span>, <span class="string">&#x27;shape&#x27;</span>, <span class="string">&#x27;data&#x27;</span>, <span class="string">&#x27;ptr&#x27;</span>))</span><br><span class="line">            logger = trt.Logger(trt.Logger.INFO)</span><br><span class="line">            <span class="keyword">with</span> <span class="built_in">open</span>(w, <span class="string">&#x27;rb&#x27;</span>) <span class="keyword">as</span> f, trt.Runtime(logger) <span class="keyword">as</span> runtime:</span><br><span class="line">                model = runtime.deserialize_cuda_engine(f.read())</span><br><span class="line">            context = model.create_execution_context()</span><br><span class="line">            bindings = OrderedDict()</span><br><span class="line">            output_names = []</span><br><span class="line">            fp16 = <span class="literal">False</span>  <span class="comment"># default updated below</span></span><br><span class="line">            dynamic = <span class="literal">False</span></span><br><span class="line">            <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(model.num_bindings):</span><br><span class="line">                name = model.get_binding_name(i)</span><br><span class="line">                dtype = trt.nptype(model.get_binding_dtype(i))</span><br><span class="line">                <span class="keyword">if</span> model.binding_is_input(i):</span><br><span class="line">                    <span class="keyword">if</span> -<span class="number">1</span> <span class="keyword">in</span> <span class="built_in">tuple</span>(model.get_binding_shape(i)):  <span class="comment"># dynamic</span></span><br><span class="line">                        dynamic = <span class="literal">True</span></span><br><span class="line">                        context.set_binding_shape(i, <span class="built_in">tuple</span>(model.get_profile_shape(<span class="number">0</span>, i)[<span class="number">2</span>]))</span><br><span class="line">                    <span class="keyword">if</span> dtype == np.float16:</span><br><span class="line">                        fp16 = <span class="literal">True</span></span><br><span class="line">                <span class="keyword">else</span>:  <span class="comment"># output</span></span><br><span class="line">                    output_names.append(name)</span><br><span class="line">                shape = <span class="built_in">tuple</span>(context.get_binding_shape(i))</span><br><span class="line">                im = torch.from_numpy(np.empty(shape, dtype=dtype)).to(device)</span><br><span class="line">                bindings[name] = Binding(name, dtype, shape, im, <span class="built_in">int</span>(im.data_ptr()))</span><br><span class="line">            binding_addrs = OrderedDict((n, d.ptr) <span class="keyword">for</span> n, d <span class="keyword">in</span> bindings.items())</span><br><span class="line">            batch_size = bindings[<span class="string">&#x27;images&#x27;</span>].shape[<span class="number">0</span>]  <span class="comment"># if dynamic, this is instead max batch size</span></span><br></pre></td></tr></table></div></figure>
<p>看不懂，过。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">    <span class="comment">// Build engine</span></span><br><span class="line">    builder-&gt;<span class="built_in">setMaxBatchSize</span>(maxBatchSize);</span><br><span class="line">    config-&gt;<span class="built_in">setMaxWorkspaceSize</span>(<span class="number">16</span> * (<span class="number">1</span> &lt;&lt; <span class="number">20</span>));  <span class="comment">// 16MB</span></span><br><span class="line"><span class="meta">#<span class="keyword">if</span> defined(USE_FP16)</span></span><br><span class="line">    config-&gt;<span class="built_in">setFlag</span>(BuilderFlag::kFP16);</span><br><span class="line"><span class="meta">#<span class="keyword">elif</span> defined(USE_INT8)</span></span><br><span class="line">    std::cout &lt;&lt; <span class="string">&quot;Your platform support int8: &quot;</span> &lt;&lt; (builder-&gt;<span class="built_in">platformHasFastInt8</span>() ? <span class="string">&quot;true&quot;</span> : <span class="string">&quot;false&quot;</span>) &lt;&lt; std::endl;</span><br><span class="line">    <span class="built_in">assert</span>(builder-&gt;<span class="built_in">platformHasFastInt8</span>());</span><br><span class="line">    config-&gt;<span class="built_in">setFlag</span>(BuilderFlag::kINT8);</span><br><span class="line">    Int8EntropyCalibrator2* calibrator = <span class="keyword">new</span> <span class="built_in">Int8EntropyCalibrator2</span>(<span class="number">1</span>, INPUT_W, INPUT_H, <span class="string">&quot;./coco_calib/&quot;</span>, <span class="string">&quot;int8calib.table&quot;</span>, INPUT_BLOB_NAME);</span><br><span class="line">    config-&gt;<span class="built_in">setInt8Calibrator</span>(calibrator);</span><br><span class="line"><span class="meta">#<span class="keyword">endif</span></span></span><br></pre></td></tr></table></div></figure>
<p>builder和config设置了一些engine的参数，然后设置了计算精度是FP16或者0INT8。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">std::cout &lt;&lt; <span class="string">&quot;Building engine, please wait for a while...&quot;</span> &lt;&lt; std::endl;</span><br><span class="line">ICudaEngine* engine = builder-&gt;<span class="built_in">buildEngineWithConfig</span>(*network, *config);</span><br><span class="line">std::cout &lt;&lt; <span class="string">&quot;Build engine successfully!&quot;</span> &lt;&lt; std::endl;</span><br></pre></td></tr></table></div></figure>
<p>build engine</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">    <span class="comment">// Don&#x27;t need the network any more</span></span><br><span class="line">    network-&gt;<span class="built_in">destroy</span>();</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Release host memory</span></span><br><span class="line">    <span class="keyword">for</span> (<span class="keyword">auto</span>&amp; mem : weightMap)</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="built_in">free</span>((<span class="type">void</span>*)(mem.second.values));</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> engine;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></div></figure>
<p>收尾工作。build_engine()全部看完。</p>
<p>回到APIToModel。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">    <span class="comment">// Serialize the engine</span></span><br><span class="line">    (*modelStream) = engine-&gt;<span class="built_in">serialize</span>();</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Close everything down</span></span><br><span class="line">    engine-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">    builder-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">    config-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">&#125;</span><br></pre></td></tr></table></div></figure>
<p>将engine序列化，然后收尾。</p>
<p>回到main()</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// create a model using the API directly and serialize it to a stream</span></span><br><span class="line"><span class="keyword">if</span> (!wts_name.<span class="built_in">empty</span>()) &#123;</span><br><span class="line">    IHostMemory* modelStream&#123; <span class="literal">nullptr</span> &#125;;</span><br><span class="line">    <span class="built_in">APIToModel</span>(BATCH_SIZE, &amp;modelStream, gd, gw, wts_name);</span><br><span class="line">    <span class="built_in">assert</span>(modelStream != <span class="literal">nullptr</span>);</span><br><span class="line">    <span class="function">std::ofstream <span class="title">p</span><span class="params">(engine_name, std::ios::binary)</span></span>;</span><br><span class="line">    <span class="keyword">if</span> (!p) &#123;</span><br><span class="line">        std::cerr &lt;&lt; <span class="string">&quot;could not open plan output file&quot;</span> &lt;&lt; std::endl;</span><br><span class="line">        <span class="keyword">return</span> <span class="number">-1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    p.<span class="built_in">write</span>(<span class="built_in">reinterpret_cast</span>&lt;<span class="type">const</span> <span class="type">char</span>*&gt;(modelStream-&gt;<span class="built_in">data</span>()), modelStream-&gt;<span class="built_in">size</span>());</span><br><span class="line">    modelStream-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></div></figure>
<p>我们仍在这一段。在将engine序列化后，可以将其导出为.engine文件保存。</p>
<p>每一次构建完网络后，都会先将其导出为.engine文件。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// deserialize the .engine and run inference</span></span><br><span class="line"><span class="function">std::ifstream <span class="title">file</span><span class="params">(engine_name, std::ios::binary)</span></span>;</span><br><span class="line"><span class="keyword">if</span> (!file.<span class="built_in">good</span>()) &#123;</span><br><span class="line">    std::cerr &lt;&lt; <span class="string">&quot;read &quot;</span> &lt;&lt; engine_name &lt;&lt; <span class="string">&quot; error!&quot;</span> &lt;&lt; std::endl;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">-1</span>;</span><br><span class="line">&#125;</span><br><span class="line"><span class="type">char</span> *trtModelStream = <span class="literal">nullptr</span>;</span><br><span class="line"><span class="type">size_t</span> size = <span class="number">0</span>;</span><br><span class="line">file.<span class="built_in">seekg</span>(<span class="number">0</span>, file.end);</span><br><span class="line">size = file.<span class="built_in">tellg</span>();</span><br><span class="line">file.<span class="built_in">seekg</span>(<span class="number">0</span>, file.beg);</span><br><span class="line">trtModelStream = <span class="keyword">new</span> <span class="type">char</span>[size];</span><br><span class="line"><span class="built_in">assert</span>(trtModelStream);</span><br><span class="line">file.<span class="built_in">read</span>(trtModelStream, size);</span><br><span class="line">file.<span class="built_in">close</span>();</span><br></pre></td></tr></table></div></figure>
<p>这一段代码就是输入.engine文件，然后将其反序列化为engine。如果已有.engine文件，就不再需要重新构建网络。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">std::vector&lt;std::string&gt; file_names;</span><br><span class="line"><span class="keyword">if</span> (<span class="built_in">read_files_in_dir</span>(img_dir.<span class="built_in">c_str</span>(), file_names) &lt; <span class="number">0</span>) &#123;</span><br><span class="line">    std::cerr &lt;&lt; <span class="string">&quot;read_files_in_dir failed.&quot;</span> &lt;&lt; std::endl;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">-1</span>;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></div></figure>
<p>TensorRTX的作者希望我们以图片的形式输入数据进行推理。当我们需要改造TensorRTX时，可以跟踪file_names的行为并将其改造为我们需要的数据流。</p>
<p>到这里为止，我们已经构建了一个用于推理的engine，接下来我们应该构建context，即申请显存。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// prepare input data ---------------------------</span></span><br><span class="line"><span class="type">static</span> <span class="type">float</span> data[BATCH_SIZE * <span class="number">3</span> * INPUT_H * INPUT_W];</span><br><span class="line"><span class="comment">//for (int i = 0; i &lt; 3 * INPUT_H * INPUT_W; i++)</span></span><br><span class="line"><span class="comment">//    data[i] = 1.0;</span></span><br><span class="line"><span class="type">static</span> <span class="type">float</span> prob[BATCH_SIZE * OUTPUT_SIZE];</span><br><span class="line">IRuntime* runtime = <span class="built_in">createInferRuntime</span>(gLogger);</span><br><span class="line"><span class="built_in">assert</span>(runtime != <span class="literal">nullptr</span>);</span><br><span class="line">ICudaEngine* engine = runtime-&gt;<span class="built_in">deserializeCudaEngine</span>(trtModelStream, size);</span><br><span class="line"><span class="built_in">assert</span>(engine != <span class="literal">nullptr</span>);</span><br><span class="line">IExecutionContext* context = engine-&gt;<span class="built_in">createExecutionContext</span>();</span><br><span class="line"><span class="built_in">assert</span>(context != <span class="literal">nullptr</span>);</span><br><span class="line"><span class="keyword">delete</span>[] trtModelStream;</span><br><span class="line"><span class="built_in">assert</span>(engine-&gt;<span class="built_in">getNbBindings</span>() == <span class="number">2</span>);</span><br><span class="line"><span class="type">void</span>* buffers[<span class="number">2</span>];</span><br><span class="line"><span class="comment">// In order to bind the buffers, we need to know the names of the input and output tensors.</span></span><br><span class="line"><span class="comment">// Note that indices are guaranteed to be less than IEngine::getNbBindings()</span></span><br><span class="line"><span class="type">const</span> <span class="type">int</span> inputIndex = engine-&gt;<span class="built_in">getBindingIndex</span>(INPUT_BLOB_NAME);</span><br><span class="line"><span class="type">const</span> <span class="type">int</span> outputIndex = engine-&gt;<span class="built_in">getBindingIndex</span>(OUTPUT_BLOB_NAME);</span><br><span class="line"><span class="built_in">assert</span>(inputIndex == <span class="number">0</span>);</span><br><span class="line"><span class="built_in">assert</span>(outputIndex == <span class="number">1</span>);</span><br><span class="line"><span class="comment">// Create GPU buffers on device</span></span><br><span class="line"><span class="built_in">CUDA_CHECK</span>(<span class="built_in">cudaMalloc</span>(&amp;buffers[inputIndex], BATCH_SIZE * <span class="number">3</span> * INPUT_H * INPUT_W * <span class="built_in">sizeof</span>(<span class="type">float</span>)));</span><br><span class="line"><span class="built_in">CUDA_CHECK</span>(<span class="built_in">cudaMalloc</span>(&amp;buffers[outputIndex], BATCH_SIZE * OUTPUT_SIZE * <span class="built_in">sizeof</span>(<span class="type">float</span>)));</span><br><span class="line"><span class="comment">// Create stream</span></span><br><span class="line">cudaStream_t stream;</span><br><span class="line"><span class="built_in">CUDA_CHECK</span>(<span class="built_in">cudaStreamCreate</span>(&amp;stream));</span><br></pre></td></tr></table></div></figure>
<p>createInferRuntime()描述为&quot;Create an instance of an IRuntime class.This class is the logging class for the runtime.&quot;。即createInferRuntime实例化的是一个logger，而ICudaEngine类描述的才是真正的engine。</p>
<p>前面从API构建网络的最后对engine进行了序列化，而读取.engine文件时也是序列化状态，因此需要反序列化deserializeCudaEngine。</p>
<p>IExecutionContext描述为&quot;使用具有功能不安全特性的引擎执行推理的上下文。一个ICudaEngine实例可能存在多个执行上下文，允许同一个引擎同时执行多个批处理。如果引擎支持动态形状，则并发使用的每个执行上下文都必须使用单独的优化配置文件。&quot;上下文实际上就是显存空间。</p>
<p>getNbBindings()，binding不知道是什么。描述为&quot;获取绑定索引的数量。如果引擎是为K个配置文件构建的，那么第一个getNbBindings()/K绑定将由配置文件编号0使用，下面的getNbBinding()/KK绑定将由第1个配置文件使用。&quot;后面用binding获取了输入和输出，猜测估计是网络中内存与显存需要进行交换的部分？？？因为除了输入输出，中间部分都可由GPU自动分配显存，但输入输出要实现约定好大小以进行数据交换。</p>
<p>cudaMalloc()和cudaStreamCreate()属于cuda编程部分，不属于TensorRT。cudaMalloc()的描述为&quot;在设备上分配线性内存的大小字节，并在*devPtr中返回分配内存的指针。分配的内存适合于任何类型的变量。内存未清除。cudaMalloc（）在失败时返回cudaErrorMemoryAllocation。“其中设备是在main()的第一行<code>cudaSetDevice(DEVICE);</code>指定的。cudaStreamCreate()的描述为&quot;Creates a new asynchronous stream.”，应该是内存与显存的传输流。</p>
<p>到了这部分，推理的准备已经全部完成，后面的部分就是推理部分。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre></td><td class="code"><pre><span class="line"><span class="type">int</span> fcount = <span class="number">0</span>;</span><br><span class="line"><span class="keyword">for</span> (<span class="type">int</span> f = <span class="number">0</span>; f &lt; (<span class="type">int</span>)file_names.<span class="built_in">size</span>(); f++) &#123;</span><br><span class="line">    fcount++;</span><br><span class="line">    <span class="keyword">if</span> (fcount &lt; BATCH_SIZE &amp;&amp; f + <span class="number">1</span> != (<span class="type">int</span>)file_names.<span class="built_in">size</span>()) <span class="keyword">continue</span>;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> b = <span class="number">0</span>; b &lt; fcount; b++) &#123;</span><br><span class="line">        cv::Mat img = cv::<span class="built_in">imread</span>(img_dir + <span class="string">&quot;/&quot;</span> + file_names[f - fcount + <span class="number">1</span> + b]);</span><br><span class="line">        <span class="keyword">if</span> (img.<span class="built_in">empty</span>()) <span class="keyword">continue</span>;</span><br><span class="line">        cv::Mat pr_img = <span class="built_in">preprocess_img</span>(img, INPUT_W, INPUT_H); <span class="comment">// letterbox BGR to RGB</span></span><br><span class="line">        <span class="type">int</span> i = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> row = <span class="number">0</span>; row &lt; INPUT_H; ++row) &#123;</span><br><span class="line">            uchar* uc_pixel = pr_img.data + row * pr_img.step;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> col = <span class="number">0</span>; col &lt; INPUT_W; ++col) &#123;</span><br><span class="line">                data[b * <span class="number">3</span> * INPUT_H * INPUT_W + i] = (<span class="type">float</span>)uc_pixel[<span class="number">2</span>] / <span class="number">255.0</span>;</span><br><span class="line">                data[b * <span class="number">3</span> * INPUT_H * INPUT_W + i + INPUT_H * INPUT_W] = (<span class="type">float</span>)uc_pixel[<span class="number">1</span>] / <span class="number">255.0</span>;</span><br><span class="line">                data[b * <span class="number">3</span> * INPUT_H * INPUT_W + i + <span class="number">2</span> * INPUT_H * INPUT_W] = (<span class="type">float</span>)uc_pixel[<span class="number">0</span>] / <span class="number">255.0</span>;</span><br><span class="line">                uc_pixel += <span class="number">3</span>;</span><br><span class="line">                ++i;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Run inference</span></span><br><span class="line">    <span class="keyword">auto</span> start = std::chrono::system_clock::<span class="built_in">now</span>();</span><br><span class="line">    <span class="built_in">doInference</span>(*context, stream, buffers, data, prob, BATCH_SIZE);</span><br><span class="line">    <span class="keyword">auto</span> end = std::chrono::system_clock::<span class="built_in">now</span>();</span><br><span class="line">    std::cout &lt;&lt; std::chrono::<span class="built_in">duration_cast</span>&lt;std::chrono::milliseconds&gt;(end - start).<span class="built_in">count</span>() &lt;&lt; <span class="string">&quot;ms&quot;</span> &lt;&lt; std::endl;</span><br><span class="line">    std::vector&lt;std::vector&lt;Yolo::Detection&gt;&gt; <span class="built_in">batch_res</span>(fcount);</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> b = <span class="number">0</span>; b &lt; fcount; b++) &#123;</span><br><span class="line">        <span class="keyword">auto</span>&amp; res = batch_res[b];</span><br><span class="line">        <span class="built_in">nms</span>(res, &amp;prob[b * OUTPUT_SIZE], CONF_THRESH, NMS_THRESH);</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> b = <span class="number">0</span>; b &lt; fcount; b++) &#123;</span><br><span class="line">        <span class="keyword">auto</span>&amp; res = batch_res[b];</span><br><span class="line">        <span class="comment">//std::cout &lt;&lt; res.size() &lt;&lt; std::endl;</span></span><br><span class="line">        cv::Mat img = cv::<span class="built_in">imread</span>(img_dir + <span class="string">&quot;/&quot;</span> + file_names[f - fcount + <span class="number">1</span> + b]);</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">size_t</span> j = <span class="number">0</span>; j &lt; res.<span class="built_in">size</span>(); j++) &#123;</span><br><span class="line">            cv::Rect r = <span class="built_in">get_rect</span>(img, res[j].bbox);</span><br><span class="line">            cv::<span class="built_in">rectangle</span>(img, r, cv::<span class="built_in">Scalar</span>(<span class="number">0x27</span>, <span class="number">0xC1</span>, <span class="number">0x36</span>), <span class="number">2</span>);</span><br><span class="line">            cv::<span class="built_in">putText</span>(img, std::<span class="built_in">to_string</span>((<span class="type">int</span>)res[j].class_id), cv::<span class="built_in">Point</span>(r.x, r.y - <span class="number">1</span>), cv::FONT_HERSHEY_PLAIN, <span class="number">1.2</span>, cv::<span class="built_in">Scalar</span>(<span class="number">0xFF</span>, <span class="number">0xFF</span>, <span class="number">0xFF</span>), <span class="number">2</span>);</span><br><span class="line">        &#125;</span><br><span class="line">        cv::<span class="built_in">imwrite</span>(<span class="string">&quot;_&quot;</span> + file_names[f - fcount + <span class="number">1</span> + b], img);</span><br><span class="line">    &#125;</span><br><span class="line">    fcount = <span class="number">0</span>;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></div></figure>
<p>基本上与pytorch的实现类似，从推理到nms，还包含计算推理时间以及最后在图上画框的部分。第一部分应该是对输入的图像进一步处理成输入数据，除了整理数据组织形式外，其他的处理操作没看懂。对TensorRTX改造时主要的处理部分。</p>
<p>其中推理接口doInference()</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="type">void</span> <span class="title">doInference</span><span class="params">(IExecutionContext&amp; context, cudaStream_t&amp; stream, <span class="type">void</span> **buffers, <span class="type">float</span>* input, <span class="type">float</span>* output, <span class="type">int</span> batchSize)</span> </span>&#123;</span><br><span class="line">    <span class="comment">// DMA input batch data to device, infer on the batch asynchronously, and DMA output back to host</span></span><br><span class="line">    <span class="built_in">CUDA_CHECK</span>(<span class="built_in">cudaMemcpyAsync</span>(buffers[<span class="number">0</span>], input, batchSize * <span class="number">3</span> * INPUT_H * INPUT_W * <span class="built_in">sizeof</span>(<span class="type">float</span>), cudaMemcpyHostToDevice, stream));</span><br><span class="line">    context.<span class="built_in">enqueue</span>(batchSize, buffers, stream, <span class="literal">nullptr</span>);</span><br><span class="line">    <span class="built_in">CUDA_CHECK</span>(<span class="built_in">cudaMemcpyAsync</span>(output, buffers[<span class="number">1</span>], batchSize * OUTPUT_SIZE * <span class="built_in">sizeof</span>(<span class="type">float</span>), cudaMemcpyDeviceToHost, stream));</span><br><span class="line">    <span class="built_in">cudaStreamSynchronize</span>(stream);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></div></figure>
<p>可以看到基本上用cuda编程，将数据送上去排队，然后等待输出。</p>
<figure class="highlight c++"><div class="table-container"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// Release stream and buffers</span></span><br><span class="line"><span class="built_in">cudaStreamDestroy</span>(stream);</span><br><span class="line"><span class="built_in">CUDA_CHECK</span>(<span class="built_in">cudaFree</span>(buffers[inputIndex]));</span><br><span class="line"><span class="built_in">CUDA_CHECK</span>(<span class="built_in">cudaFree</span>(buffers[outputIndex]));</span><br><span class="line"><span class="comment">// Destroy the engine</span></span><br><span class="line">context-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">engine-&gt;<span class="built_in">destroy</span>();</span><br><span class="line">runtime-&gt;<span class="built_in">destroy</span>();</span><br></pre></td></tr></table></div></figure>
<p>最后的收尾工作，包括传输流的回收，释放显存，最后释放内存。</p>
</div><footer class="post-footer"><div class="post-ending ending"><div class="ending__text">------ 本文结束，感谢您的阅读 ------</div></div><div class="post-copyright copyright"><div class="copyright-author"><span class="copyright-author__name">本文作者: </span><span class="copyright-author__value"><a href="http://hipposox.github.io">HippoSoX</a></span></div><div class="copyright-link"><span class="copyright-link__name">本文链接: </span><span class="copyright-link__value"><a href="http://hipposox.github.io/2023/01/12/TensorRT/">http://hipposox.github.io/2023/01/12/TensorRT/</a></span></div><div class="copyright-notice"><span class="copyright-notice__name">版权声明: </span><span class="copyright-notice__value">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/deed.en" rel="external nofollow" target="_blank">BY-NC-SA</a> 许可协议。转载请注明出处！</span></div></div><div class="post-tags"><span class="post-tags-item"><span class="post-tags-item__icon"><i class="fas fa-tag"></i></span><a class="post-tags-item__link" href="http://hipposox.github.io/tags/TensorRT/">TensorRT</a></span><span class="post-tags-item"><span class="post-tags-item__icon"><i class="fas fa-tag"></i></span><a class="post-tags-item__link" href="http://hipposox.github.io/tags/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C/">神经网络</a></span></div><nav class="post-paginator paginator"><div class="paginator-prev"><a class="paginator-prev__link" href="/2023/01/12/OCCT-Tutorials-and-Demos-01-Novice-Guide/"><span class="paginator-prev__icon"><i class="fas fa-angle-left"></i></span><span class="paginator-prev__text">OCCT-Tutorials_and_Demos_01:Novice_Guide</span></a></div><div class="paginator-next"><a class="paginator-next__link" href="/2023/01/11/OCCT-Introduction/"><span class="paginator-prev__text">OCCT_Introduction</span><span class="paginator-next__icon"><i class="fas fa-angle-right"></i></span></a></div></nav></footer></div></div></div><div class="sidebar-wrap" id="sidebar-wrap"><aside class="sidebar" id="sidebar"><div class="sidebar-nav"><span class="sidebar-nav-toc current">文章目录</span><span class="sidebar-nav-ov">站点概览</span></div><section class="sidebar-toc"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E8%AF%BB%E4%BB%A3%E7%A0%81%E5%AD%A6%E4%B9%A0tensorrt"><span class="toc-number">1.</span> <span class="toc-text">
           读代码学习TensorRT</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#tensorrt-%E5%81%9A%E7%9A%84%E5%B7%A5%E4%BD%9C"><span class="toc-number">1.1.</span> <span class="toc-text">
           TensorRT 做的工作</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E7%89%88%E6%9C%AC"><span class="toc-number">1.2.</span> <span class="toc-text">
           版本</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#yolov5cpp"><span class="toc-number">1.3.</span> <span class="toc-text">
           yolov5.cpp</span></a></li></ol></li></ol></section><!-- ov = overview--><section class="sidebar-ov hide"><div class="sidebar-ov-author"><div class="sidebar-ov-author__avatar"><img class="sidebar-ov-author__avatar_img" src="/images/icons/stun-logo.svg" alt="avatar"></div><p class="sidebar-ov-author__text">motto</p></div><div class="sidebar-ov-state"><a class="sidebar-ov-state-item sidebar-ov-state-item--posts" href="/archives/"><div class="sidebar-ov-state-item__count">19</div><div class="sidebar-ov-state-item__name">归档</div></a><a class="sidebar-ov-state-item sidebar-ov-state-item--categories" href="/categories/"><div class="sidebar-ov-state-item__count">6</div><div class="sidebar-ov-state-item__name">分类</div></a><a class="sidebar-ov-state-item sidebar-ov-state-item--tags" href="/tags/"><div class="sidebar-ov-state-item__count">14</div><div class="sidebar-ov-state-item__name">标签</div></a></div><div class="sidebar-ov-cc"><a href="https://creativecommons.org/licenses/by-nc-sa/4.0/deed.en" target="_blank" rel="noopener" data-popover="知识共享许可协议" data-popover-pos="up"><img src="/images/cc-by-nc-sa.svg"></a></div></section><div class="sidebar-reading"><div class="sidebar-reading-info"><span class="sidebar-reading-info__text">你已阅读了 </span><span class="sidebar-reading-info__num">0</span><span class="sidebar-reading-info__perc">%</span></div><div class="sidebar-reading-line"></div></div><iframe frameborder="no" border="0" marginwidth="0" marginheight="0" width=330 height=86 src="//music.163.com/outchain/player?type=2&id=1449790718&auto=1&height=66"></iframe></aside></div><div class="clearfix"></div></div></main><footer class="footer" id="footer"><div class="footer-inner"><div><span>Copyright © 2023</span><span class="footer__icon"><i class="fas fa-heart"></i></span><span>HippoSoX</span></div><div><span>由 <a href="http://hexo.io/" title="Hexo" target="_blank" rel="noopener">Hexo</a> 强力驱动</span><span> v5.4.2</span><span class="footer__devider">|</span><span>主题 - <a href="https://github.com/liuyib/hexo-theme-stun/" title="Stun" target="_blank" rel="noopener">Stun</a></span><span> v2.8.0</span></div></div></footer><div class="loading-bar" id="loading-bar"><div class="loading-bar__progress"></div></div><div class="back2top" id="back2top"><span class="back2top__icon"><i class="fas fa-rocket"></i></span></div></div><script src="https://cdn.jsdelivr.net/npm/jquery@v3.4.1/dist/jquery.min.js"></script><script src="https://cdn.jsdelivr.net/npm/velocity-animate@1.5.2/velocity.min.js"></script><script src="https://cdn.jsdelivr.net/npm/velocity-animate@1.5.2/velocity.ui.min.js"></script><script src="https://cdn.jsdelivr.net/npm/lazyload@2.0.0-rc.2/lazyload.min.js"></script><script src="/js/utils.js?v=2.8.0"></script><script src="/js/stun-boot.js?v=2.8.0"></script><script src="/js/scroll.js?v=2.8.0"></script><script src="/js/header.js?v=2.8.0"></script><script src="/js/sidebar.js?v=2.8.0"></script></body></html>